package fetcher

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"time"

	"golang.org/x/net/html/charset"
	"golang.org/x/text/encoding"
	"golang.org/x/text/encoding/unicode"
	"golang.org/x/text/transform"

	"gitee.com/jnshao/go-start/projects/crawler_distributed/config"
)

var rateLimiter = time.Tick(time.Second / config.Qps)

// 获取网络数据
func Fetch(url string) ([]byte, error) {
	<-rateLimiter
	log.Printf("Fetching url %s", url)
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("wrong status code: %d", resp.StatusCode)
	}

	bodyReader := bufio.NewReader(resp.Body)
	encoding := determineEncoding(bodyReader)
	utf8Reader := transform.NewReader(bodyReader, encoding.NewDecoder()) //转utf8编码阅读器

	return ioutil.ReadAll(utf8Reader)
}

// 获取html编码
func determineEncoding(r *bufio.Reader) encoding.Encoding { //参数必须是*bufio.Reader
	//bytes, err := bufio.NewReader(r).Peek(1024) //对bufio peek:对底下r读取1024存到缓存
	//此时resp.Body的位置已经变了，transform.NewReader()在读数据会缺失
	bytes, err := r.Peek(1024)
	if err != nil {
		log.Printf("Fetcher error: %v", err)
		return unicode.UTF8
	}
	e, _, _ := charset.DetermineEncoding(bytes, "")
	return e
}
